package com.proxyhub.crawler;

import com.proxyhub.annotation.Crawler;
import com.proxyhub.model.Proxy;
import com.proxyhub.repo.ProxyRepo;
import com.proxyhub.utils.CommonUtil;
import com.proxyhub.utils.ProxyUtil;
import com.proxyhub.utils.RequestUtil;
import org.apache.commons.lang3.math.NumberUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

/**
 * 站大爷代理抓取:免费用户只能查看第一页
 * Created by linxsh on 2017/6/15.
 */
@Crawler("站大爷")
@Service
public class ZdayeCrawler extends BaseCrawler {
    private static final String seedUrl = "http://ip.zdaye.com/";
    private static final int[] commonPorts = new int[] {
        80, 8080, 8081, 3128, 1189, 808, 9000, 8998, 1080
    };

    @Override
    public void crawl() {
        Document firstPage = RequestUtil.doGetDoc(seedUrl, CommonUtil.GBK);
        if(firstPage != null) {
            //解析页面，获取代理
            List<Proxy> proxyList = parsePage(firstPage);
            //过滤有用的代理IP
            filterValidProxy(proxyList);
            //入库
            proxyRepo.insertIfNotExist(proxyList);
        }
    }

    /**
     * 过滤无用的IP
     * @param proxyList
     */
    private void filterValidProxy(List<Proxy> proxyList) {
        /*if(!CollectionUtils.isEmpty(proxyList)) {
            Iterator<Proxy> it = proxyList.iterator();
            while (it.hasNext()) {
                Proxy proxy = it.next();

            }
        }*/
    }

    /**
     * 解析页面
     * @param firstPage
     * @return
     */
    private List<Proxy> parsePage(Document firstPage) {
        if(firstPage == null) {
            return Collections.emptyList();
        }
        Elements trs = firstPage.select("#ipc tbody tr:gt(0)");
        List<Proxy> proxies = new ArrayList<Proxy>();
        for(Element tr : trs) {
            String host = tr.select("td:eq(0)").text();
            String type = tr.select("td:eq(3)").text();
            String addr = tr.select("td:eq(4)").text();
            boolean https = tr.select("td:eq(6)").first().children().size() > 0;
            String delay = tr.select("td:eq(18)").text().replaceAll("[^.\\d]", "");

            if(!CommonUtil.isIP(host)) {
                continue;
            }
            Integer port = getPort(host);
            if(port == null) {
                continue;
            }
            proxies.add(new Proxy(host, port, type, addr, https, NumberUtils.toInt(delay), true));
        }
        return proxies;
    }

    /**
     * 尝试获取可用端口
     * @param host
     */
    private Integer getPort(String host) {
        for(int port : commonPorts) {
            if(ProxyUtil.validateProxy(host, port)) {
                return port;
            }
        }
        return null;
    }
}
